Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Customize and use fix_encumbrances_quesnelia script #1494

Closed
wants to merge 1 commit into from
Closed

Customize and use fix_encumbrances_quesnelia script #1494

wants to merge 1 commit into from

Conversation

jgreben
Copy link
Contributor

@jgreben jgreben commented Jan 30, 2025

To be merged after we upgrade to Quesnelia.

@jgreben
Copy link
Contributor Author

jgreben commented Jan 30, 2025 •
edited
Loading

This is the diff after all of our customizations were applied to the new script. Some of it is just formatting differences:

--- libsys_airflow/plugins/folio/encumbrances/fix_encumbrances_quesnelia.py	2025-01-30 10:50:19
+++ libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py	2024-12-17 16:21:51
@@ -26,7 +26,7 @@
 dryrun = False
 
 # request timeout in seconds
-ASYNC_CLIENT_TIMEOUT = 30
+ASYNC_CLIENT_TIMEOUT = 60
 
 # limit the number of parallel threads.
 # Try different values. Bigger values - for increasing performance, but could produce "Connection timeout exception"
@@ -45,7 +45,7 @@
     login_headers = {'x-okapi-tenant': tenant, 'Content-Type': 'application/json'}
     data = {'username': username, 'password': password}
     try:
-        r = requests.post(okapi_url + 'authn/login', headers=login_headers, json=data)
+        r = requests.post(okapi_url + '/authn/login', headers=login_headers, json=data)
         if r.status_code != 201:
             raise_exception_for_reply(r)
         logger.info('Logged in successfully.')
@@ -97,44 +97,39 @@
         raise Exception("Exiting Fix Encumbrances script.")
 
 
-async def post_request(url: str, data):
+async def put_request(url: str, data):
     if dryrun:
         return
     try:
-        resp = await client.post(
+        resp = await client.put(
             url, headers=headers, data=json.dumps(data), timeout=ASYNC_CLIENT_TIMEOUT
         )
-        if (
-            resp.status_code == HTTPStatus.CREATED
-            or resp.status_code == HTTPStatus.NO_CONTENT
-        ):
+        if resp.status_code == HTTPStatus.NO_CONTENT:
             return
-        print(f'Error in POST request {url} "{data}": {resp.text}')
-        logger.error(f'Error in POST request {url} "{data}": {resp.text}')
+        print(f'Error updating record {url} "{data}": {resp.text}')
+        logger.error(f'Error updating record {url} "{data}": {resp.text}')
         raise Exception("Exiting Fix Encumbrances script.")
 
     except Exception as err:
-        print(f'Error in POST request {url} "{data}": {err=}')
-        logger.error(f'Error in POST request {url} "{data}": {err=}')
+        print(f'Error updating record {url} "{data}": {err=}')
+        logger.error(f'Error updating record {url} "{data}": {err=}')
         raise Exception("Exiting Fix Encumbrances script.")
 
 
-async def put_request(url: str, data):
+async def delete_request(url: str):
     if dryrun:
         return
     try:
-        resp = await client.put(
-            url, headers=headers, data=json.dumps(data), timeout=ASYNC_CLIENT_TIMEOUT
-        )
+        resp = await client.delete(url, headers=headers, timeout=ASYNC_CLIENT_TIMEOUT)
         if resp.status_code == HTTPStatus.NO_CONTENT:
             return
-        print(f'Error updating record {url} "{data}": {resp.text}')
-        logger.error(f'Error updating record {url} "{data}": {resp.text}')
+        print(f'Error deleting record {url}: {resp.text}')
+        logger.error(f'Error deleting record {url}: {resp.text}')
         raise Exception("Exiting Fix Encumbrances script.")
 
     except Exception as err:
-        print(f'Error updating record {url} "{data}": {err=}')
-        logger.error(f'Error updating record {url} "{data}": {err=}')
+        print(f'Error deleting record {url}: {err=}')
+        logger.error(f'Error deleting record {url}: {err=}')
         raise Exception("Exiting Fix Encumbrances script.")
 
 
@@ -142,7 +137,7 @@
     params = {'query': query, 'offset': '0', 'limit': ITEM_MAX}
     try:
         r = requests.get(
-            okapi_url + 'finance-storage/fiscal-years', headers=headers, params=params
+            okapi_url + '/finance-storage/fiscal-years', headers=headers, params=params
         )
         if r.status_code != 200:
             raise_exception_for_reply(r)
@@ -182,7 +177,7 @@
 def get_order_ids_by_query(query) -> list:
     try:
         orders = get_by_chunks(
-            okapi_url + 'orders-storage/purchase-orders', query, 'purchaseOrders'
+            okapi_url + '/orders-storage/purchase-orders', query, 'purchaseOrders'
         )
         ids = []
         for order in orders:
@@ -195,7 +190,7 @@
 
 
 async def get_encumbrances_by_query(query) -> list:
-    url = okapi_url + 'finance-storage/transactions'
+    url = okapi_url + '/finance-storage/transactions'
     response = await get_request(url, query)
     return response['transactions']
 
@@ -207,7 +202,7 @@
             query = query + f"id=={enc_id} OR "
         else:
             query = query + f"id=={enc_id}"
-    resp = await get_request(okapi_url + 'finance-storage/transactions', query)
+    resp = await get_request(okapi_url + '/finance-storage/transactions', query)
 
     return resp['transactions']
 
@@ -233,7 +228,10 @@
     print('Retrieving closed order ids...')
     query = 'workflowStatus=="Closed"'
     closed_orders_ids = get_order_ids_by_query(query)
-    print('  Closed orders:', len(closed_orders_ids))
+    if len(closed_orders_ids):
+        print('  Closed orders:', len(closed_orders_ids))
+    else:
+        print('No closed orders')
     return closed_orders_ids
 
 
@@ -241,24 +239,21 @@
     print('Retrieving open order ids...')
     query = 'workflowStatus=="Open"'
     open_orders_ids = get_order_ids_by_query(query)
-    print('  Open orders:', len(open_orders_ids))
+    if len(open_orders_ids):
+        print('  Open orders:', len(open_orders_ids))
+    else:
+        print('No open orders')
     return open_orders_ids
 
 
-async def batch_update(transactions: list):
-    batch = {'transactionsToUpdate': transactions}
-    url = f'{okapi_url}finance-storage/transactions/batch-all-or-nothing'
-    await post_request(url, batch)
+async def transaction_summary(order_id, num_transactions):
+    data = {'id': order_id, 'numTransactions': num_transactions}
+    url = f'{okapi_url}/finance-storage/order-transaction-summaries/{order_id}'
+    await put_request(url, data)
 
 
-async def batch_delete(transaction_ids: list):
-    batch = {'idsOfTransactionsToDelete': transaction_ids}
-    url = f'{okapi_url}finance-storage/transactions/batch-all-or-nothing'
-    await post_request(url, batch)
-
-
 async def get_budgets_by_query(query) -> list:
-    budget_collection = await get_request(okapi_url + 'finance-storage/budgets', query)
+    budget_collection = await get_request(okapi_url + '/finance-storage/budgets', query)
     return budget_collection['budgets']
 
 
@@ -282,7 +277,7 @@
 
 
 async def get_order_encumbrances(order_id, fiscal_year_id, sem=None) -> list:
-    url = okapi_url + 'finance-storage/transactions'
+    url = okapi_url + '/finance-storage/transactions'
     query = f'encumbrance.sourcePurchaseOrderId=={order_id} AND fiscalYearId=={fiscal_year_id}'
     response = await get_request(url, query)
     if sem is not None:
@@ -351,7 +346,7 @@
 async def update_poline_encumbrance(encumbrance_to_remove, replace_by, poline=None):
     url = (
         okapi_url
-        + f"orders-storage/po-lines/{encumbrance_to_remove['encumbrance']['sourcePoLineId']}"
+        + f"/orders-storage/po-lines/{encumbrance_to_remove['encumbrance']['sourcePoLineId']}"
     )
     if poline is None:
         poline = await get_request_without_query(url)
@@ -364,7 +359,6 @@
 
 async def remove_encumbrances_and_update_polines(encumbrance_changes):
     futures = []
-    ids_to_delete = []
     for change in encumbrance_changes:
         encumbrance_to_remove = change['remove']
         replace_by = change['replace_by']
@@ -373,9 +367,9 @@
                 update_poline_encumbrance(encumbrance_to_remove, replace_by)
             )
         )
-        ids_to_delete.append(encumbrance_to_remove['id'])
+        url = f"{okapi_url}/finance-storage/transactions/{encumbrance_to_remove['id']}"
+        futures.append(asyncio.ensure_future(delete_request(url)))
     await asyncio.gather(*futures)
-    await batch_delete(ids_to_delete)
 
 
 async def remove_duplicate_encumbrances_in_order(order_id, fiscal_year_id, sem) -> int:
@@ -401,7 +395,7 @@
     sem = asyncio.Semaphore(MAX_ACTIVE_THREADS)
     for idx, order_id in enumerate(open_and_closed_orders_ids):
         await sem.acquire()
-        progress(idx, len(open_and_closed_orders_ids))
+        # progress(idx, len(open_and_closed_orders_ids))
         futures.append(
             asyncio.ensure_future(
                 remove_duplicate_encumbrances_in_order(order_id, fiscal_year_id, sem)
@@ -409,7 +403,7 @@
         )
 
     nb_removed_encumbrances = await asyncio.gather(*futures)
-    progress(len(open_and_closed_orders_ids), len(open_and_closed_orders_ids))
+    # progress(len(open_and_closed_orders_ids), len(open_and_closed_orders_ids))
     print(f'  Removed {sum(nb_removed_encumbrances)} encumbrance(s).')
 
 
@@ -419,7 +413,7 @@
 
 async def get_polines_by_order_id(order_id) -> list:
     query = f'purchaseOrderId=={order_id}'
-    resp = await get_request(okapi_url + 'orders-storage/po-lines', query)
+    resp = await get_request(okapi_url + '/orders-storage/po-lines', query)
     po_lines = resp['poLines']
     return po_lines
 
@@ -427,10 +421,13 @@
 async def update_encumbrance_fund_id(encumbrance, new_fund_id, poline):
     encumbrance['fromFundId'] = new_fund_id
     encumbrance_id = encumbrance['id']
+    order_id = poline['purchaseOrderId']
     print(
         f"  Fixing fromFundId for po line {poline['id']} ({poline['poLineNumber']}) encumbrance {encumbrance_id}"
     )
-    await batch_update([encumbrance])
+    await transaction_summary(order_id, 1)
+    url = f"{okapi_url}/finance-storage/transactions/{encumbrance_id}"
+    await put_request(url, encumbrance)
 
 
 # Remove a duplicate encumbrance if it has a wrong fromFundId, and update the poline fd if needed
@@ -455,15 +452,14 @@
         )
         return
     replace_by = encumbrances_with_right_fund[0]
-    ids_to_delete = []
     for encumbrance_to_remove in encumbrances_with_bad_fund:
         print(
             f"  Removing encumbrance {encumbrance_to_remove['id']} for po line {poline['id']} "
             f"({poline['poLineNumber']})"
         )
         await update_poline_encumbrance(encumbrance_to_remove, replace_by, poline)
-        ids_to_delete.append(encumbrance_to_remove['id'])
-    await batch_delete(ids_to_delete)
+        url = f"{okapi_url}/finance-storage/transactions/{encumbrance_to_remove['id']}"
+        await delete_request(url)
 
 
 # Fix encumbrance fromFundId if it doesn't match the po line fund distribution (see MODFISTO-384, MODFISTO-385)
@@ -492,11 +488,7 @@
     poline_id = poline['id']
     fd_fund_id = fd['fundId']
     for enc in order_encumbrances:
-        # if (
-        #     enc['encumbrance']['sourcePoLineId'] == poline_id
-        #     and float(enc['amount']) != 0.0
-        #     and enc['fromFundId'] == fd_fund_id
-        # ):
+        # if enc['encumbrance']['sourcePoLineId'] == poline_id and float(enc['amount']) != 0.0 and \
         if (
             enc['encumbrance']['sourcePoLineId'] == poline_id
             and enc['fromFundId'] == fd_fund_id
@@ -504,10 +496,8 @@
             fd_encumbrance_id = fd['encumbrance']
             if enc['id'] == fd_encumbrance_id:
                 return False
-            # print(
-            #     f"  Updating poline {poline_id} ({poline['poLineNumber']}) encumbrance {fd_encumbrance_id} "
-            #     f"with new value {enc['id']}"
-            # )
+            # print(f"  Updating poline {poline_id} ({poline['poLineNumber']}) encumbrance {fd_encumbrance_id} "
+            #       f"with new value {enc['id']}")
             fd['encumbrance'] = enc['id']
             return True
     return False
@@ -528,7 +518,7 @@
 
     # update poline if one or more fund distributions modified
     if poline_needs_updates:
-        url = f"{okapi_url}orders-storage/po-lines/{poline['id']}"
+        url = f"{okapi_url}/orders-storage/po-lines/{poline['id']}"
         await put_request(url, poline)
 
 
@@ -571,7 +561,7 @@
     order_sem = asyncio.Semaphore(MAX_ACTIVE_THREADS)
     for idx, order_id in enumerate(open_orders_ids):
         await order_sem.acquire()
-        progress(idx, len(open_orders_ids))
+        # progress(idx, len(open_orders_ids))
         orders_futures.append(
             asyncio.ensure_future(
                 process_order_encumbrances_relations(
@@ -580,7 +570,7 @@
             )
         )
     await asyncio.gather(*orders_futures)
-    progress(len(open_orders_ids), len(open_orders_ids))
+    # progress(len(open_orders_ids), len(open_orders_ids))
 
 
 # ---------------------------------------------------
@@ -592,15 +582,18 @@
         f'encumbrance.orderStatus<>"Closed" AND encumbrance.sourcePurchaseOrderId=={order_id} AND '
         f'fiscalYearId=={fiscal_year_id}'
     )
-    url = okapi_url + 'finance-storage/transactions'
+    url = okapi_url + '/finance-storage/transactions'
 
     return await get_request(url, query)
 
 
-async def unrelease_order_encumbrances(encumbrances) -> list:
+async def unrelease_order_encumbrances(order_id, encumbrances) -> list:
+    await transaction_summary(order_id, len(encumbrances))
+
     for encumbrance in encumbrances:
         encumbrance['encumbrance']['status'] = 'Unreleased'
-    await batch_update(encumbrances)
+        url = f"{okapi_url}/finance-storage/transactions/{encumbrance['id']}"
+        await put_request(url, encumbrance)
 
     # the encumbrance amounts get modified (and the version in MG), so we need to get a fresh version
     modified_encumbrance_futures = []
@@ -637,10 +630,13 @@
 
 async def fix_order_status_and_release_encumbrances(order_id, encumbrances):
     try:
+        await transaction_summary(order_id, len(encumbrances))
+
         for encumbrance in encumbrances:
             encumbrance['encumbrance']['status'] = 'Released'
             encumbrance['encumbrance']['orderStatus'] = 'Closed'
-        await batch_update(encumbrances)
+            url = f"{okapi_url}/finance-storage/transactions/{encumbrance['id']}"
+            await put_request(url, encumbrance)
 
     except Exception as err:
         print(
@@ -662,7 +658,9 @@
         # print(f'\n  Fixing the following encumbrance(s) for order {order_id} :')
         for encumbrance in encumbrances:
             print(f"    {encumbrance['id']}")
-        modified_encumbrances = await unrelease_order_encumbrances(encumbrances)
+        modified_encumbrances = await unrelease_order_encumbrances(
+            order_id, encumbrances
+        )
         if len(modified_encumbrances) != 0:
             await fix_order_status_and_release_encumbrances(
                 order_id, modified_encumbrances
@@ -720,8 +718,13 @@
     # print(f'\n  Unreleasing the following encumbrance(s) for order {order_id} :')
     for encumbrance in encumbrances:
         print(f"    {encumbrance['id']}")
+
+    await transaction_summary(order_id, len(encumbrances))
+
+    for encumbrance in encumbrances:
         encumbrance['encumbrance']['status'] = 'Unreleased'
-    await batch_update(encumbrances)
+        url = f"{okapi_url}/finance-storage/transactions/{encumbrance['id']}"
+        await put_request(url, encumbrance)
 
 
 async def unrelease_encumbrances_with_non_zero_amounts(
@@ -732,7 +735,7 @@
         f'fiscalYearId=={fiscal_year_id}'
     )
     transactions_response = await get_request(
-        okapi_url + 'finance-storage/transactions', query
+        okapi_url + '/finance-storage/transactions', query
     )
 
     order_encumbrances = transactions_response['transactions']
@@ -776,11 +779,16 @@
 
 
 async def release_encumbrances(order_id, encumbrances):
-    # print(f'\n  Releasing the following encumbrances for order {order_id} :')
+    # print\(f'\n  Releasing the following encumbrances for order {order_id} :')
     for encumbrance in encumbrances:
         print(f"    {encumbrance['id']}")
+
+    await transaction_summary(order_id, len(encumbrances))
+
+    for encumbrance in encumbrances:
         encumbrance['encumbrance']['status'] = 'Released'
-    await batch_update(encumbrances)
+        url = f"{okapi_url}/finance-storage/transactions/{encumbrance['id']}"
+        await put_request(url, encumbrance)
 
 
 async def release_encumbrances_with_negative_amounts(
@@ -792,7 +800,7 @@
         f'encumbrance.sourcePurchaseOrderId=={order_id} AND fiscalYearId=={fiscal_year_id}'
     )
     transactions_response = await get_request(
-        okapi_url + 'finance-storage/transactions', query
+        okapi_url + '/finance-storage/transactions', query
     )
 
     order_encumbrances = transactions_response['transactions']
@@ -899,12 +907,10 @@
 
     # Cast into decimal values, so 0 == 0.0 == 0.00 will return true
     if Decimal(str(budget['encumbered'])) != Decimal(encumbered):
-        # print(
-        #     f"    Budget \"{budget['name']}\": changing encumbered from {budget['encumbered']} to {encumbered}"
-        # )
+        # print(f"    Budget \"{budget['name']}\": changing encumbered from {budget['encumbered']} to {encumbered}")
         budget['encumbered'] = encumbered
 
-        url = f"{okapi_url}finance-storage/budgets/{budget['id']}"
+        url = f"{okapi_url}/finance-storage/budgets/{budget['id']}"
         await put_request(url, budget)
         nb_modified = 1
     sem.release()
@@ -1016,10 +1022,9 @@
     await fix_poline_encumbrances_relations(
         open_orders_ids, fiscal_year_id, fy_is_current
     )
-    if fy_is_current:
-        await fix_encumbrance_order_status_for_closed_orders(
-            closed_orders_ids, fiscal_year_id
-        )
+    await fix_encumbrance_order_status_for_closed_orders(
+        closed_orders_ids, fiscal_year_id
+    )
     await unrelease_open_orders_encumbrances_with_nonzero_amounts(
         fiscal_year_id, open_orders_ids
     )

@jgreben
Copy link
Contributor Author

jgreben commented Feb 19, 2025

Messed up after rebase.

@jgreben jgreben closed this Feb 19, 2025
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Reviewers

@shelleydoljack shelleydoljack Awaiting requested review from shelleydoljack

@jermnelson jermnelson Awaiting requested review from jermnelson

Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
No milestone
Development

Successfully merging this pull request may close these issues.
1 participant
@jgreben